{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "import json\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefs = {'profile.default_content_settings.popups': 0, \n",
    "         'profile.default_content_settings.automaticDownloads': 0, \n",
    "         'download.default_directory': 'I:\\\\情报通文件\\\\',\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "options = webdriver.ChromeOptions()\n",
    "options.add_argument('disable-infobars')\n",
    "options.add_argument('lang=zh_CN.UTF-8')\n",
    "options.add_experimental_option('prefs', prefs)\n",
    "options.add_extension(r\"D:\\Python\\IPython\\information\\DogAuthExtension.crx\")\n",
    "driver = webdriver.Chrome(chrome_options=options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(r'http://qbt.ecdataway.com/site/login')\n",
    "driver.find_element_by_id('LoginForm_username').send_keys('森马上海服饰')\n",
    "driver.implicitly_wait(10)\n",
    "driver.find_element_by_id('LoginForm_password').send_keys('a123')\n",
    "driver.find_element_by_class_name('login_btn').click()\n",
    "driver.implicitly_wait(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "locator = (By.ID, 'header')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(r'http://qbt.ecdataway.com/stat#fold_line')\n",
    "try:\n",
    "    ele = WebDriverWait(driver, 30).until(EC.presence_of_element_located(locator))\n",
    "    driver.implicitly_wait(5)\n",
    "    driver.find_element_by_css_selector('#sidebar_category > ul > li:nth-child(4) > a').click()\n",
    "except:\n",
    "    print('time is short and  need take long time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_one_url():\n",
    "    with open(r\"I:\\情报通\\链接.txt\", 'r', encoding='utf-8') as f:\n",
    "        for one_url in f.readlines():\n",
    "            yield one_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# url_element = []\n",
    "# for one_url in get_one_url():\n",
    "#     driver.get(one_url)\n",
    "#     try:\n",
    "#         ele = WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator))\n",
    "#         driver.implicitly_wait(30)\n",
    "#         time.sleep(5)\n",
    "#         all_href = driver.find_elements_by_xpath(\"//*[@href]\")\n",
    "#     except:\n",
    "#         print('time is short and  need take long time')\n",
    "#     element = []\n",
    "#     for link in all_href:\n",
    "#         href = link.get_attribute('href')\n",
    "#         if 'cid' in href and 'pos' in href:\n",
    "#             print(link.text, href)\n",
    "#             element.append((link.text, href))\n",
    "#     url_element.append((one_url, element))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = pd.DataFrame(url_element)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd.DataFrame(df[1][0])\n",
    "# df[0].str.strip('\\n')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dic_url_element"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lst_df = []\n",
    "# for i in range(len(df)):\n",
    "#     one = pd.DataFrame(df[1][i])\n",
    "#     one['url'] = df[0].str.strip('\\n')[i]\n",
    "#     lst_df.append(one)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_url = pd.concat(lst_df, ignore_index=False,join='outer', sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_url.to_excel('I:\\\\所有元素链接.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# classfaction = pd.read_excel(r\"E:\\文档\\爬虫\\情报通分类.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# classfaction.drop('元素',axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new = pd.merge(classfaction, all_url, how='outer', left_on='URL', right_on='url')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new.to_excel('I:\\\\element_url.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium.webdriver.support.ui import Select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(r'http://qbt.ecdataway.com/stat?cid=162201&pos=19#fold_line')\n",
    "try:\n",
    "    ele = WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator))\n",
    "    driver.implicitly_wait(30)\n",
    "    time.sleep(5)\n",
    "except:\n",
    "    print('time is short and  need take long time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#content > div.charts > div > div.submenu > a:nth-child(2) > span.body').click()\n",
    "try:\n",
    "    ele = WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator))\n",
    "    driver.implicitly_wait(30)\n",
    "    time.sleep(1)\n",
    "except:\n",
    "    print('time is short and  need take long time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "period_start = Select(driver.find_element_by_id('period_start'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "period_start.select_by_value('201707')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "period_last = Select(driver.find_element_by_id('period_last'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "period_last.select_by_value('201712')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_name('is_mall').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#f1 > table > tbody > tr:nth-child(6) > td:nth-child(2) > input:nth-child(2)').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "locator_set1 = (By.ID, 'set1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_id('submit-button').click()\n",
    "try:\n",
    "    ele = WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator_set1))\n",
    "    driver.implicitly_wait(30)\n",
    "    time.sleep(5)\n",
    "except:\n",
    "    print('check failed and need take long time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#content > div.charts > div > div.padding-15 > p > button').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-99-24212d8309f2>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-99-24212d8309f2>\"\u001b[1;36m, line \u001b[1;32m1\u001b[0m\n\u001b[1;33m    driver.execute_script('document.getElementById('is_price').value=1;document.getElementById('is_csv').value=1;document.f1.submit();')\u001b[0m\n\u001b[1;37m                                                           ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "js = '''document.getElementById('is_price').value=1;document.getElementById('is_csv').value=1;document.f1.submit();'''\n",
    "driver.execute_script(js)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
